#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# This program is free software; you can redistribute it and/or modify  
# it under the terms of the GNU General Public License as published by  
# the Free Software Foundation; either version 2 of the License, or     
# (at your option) any later version.                                   
#                                                                         
# A copy of the license can be found in the license.txt file supplied   
# with this software or at: http://www.gnu.org/copyleft/gpl.html       
#


import celtuce.help.logging as logging
import celtuce.help.codec as codec

import celtuce.net.ioloop as ioloop
import celtuce.net.iostream as iostream

from celtuce.event import Event,ROUTE_GET,ROUTE_DIGEST,TS_HEART_BEAT,OP_RESPONSE, OP_TEST_ECHO
from celtuce.event import OP_INSERT,OP_ERASE,OP_FIND,OP_INSERTUNIQUE,OP_UPDATE,OP_UPSERT
from celtuce.response import Response,RC_TIMEOUT,RC_INTERNAL_ERROR
from celtuce.task import Task,TASK_CREATE_TABLET,TASK_DROP_TABLET

import celtuce.help.url as urlparser

from storer import Storer


import Queue as queue
import time, errno, socket, fcntl
import ConfigParser as configparser

try :
    from hashlib import md5 as MD5
except :
    from md5 import md5 as MD5
from bisect import bisect_left,insort
import base64

class   Tcpio(object) :
    HEART_BEAT_INTERVAL = 10

    def __init__(self, request_callable, tcp_port, meta_url, hb_response_callback, hb_data_callback) :
        self._tcp_server = None

        self._io_loop = ioloop.IOLoop()

        if tcp_port :
            self._tcp_port = tcp_port
            self._tcp_server = TcpServer(request_callable, self._io_loop)

        self._meta_url = meta_url
        parser = urlparser.URL(meta_url)
        self._meta_protocol = parser.protocol
        self._meta_host = parser.host
        self._meta_port = parser.port

        self.__handleHeartBeatResponse = hb_response_callback
        self.__getHeartBeatData = hb_data_callback
        self._heart_beat_interval = 1

        self._END_TAG = "\r\n\r\n"

        deadline = time.time() + self._heart_beat_interval
        self._heart_beat_handler = self._io_loop.add_timeout(deadline, self.__onHeartBeat)

        self._logger = logging.getLogger('celtuce')
        ## __init__()

    def run(self) :
        self._tcp_server.listen( self._tcp_port )

        self._io_loop.start()
        ## run()

    def stop(self) :
        self._io_loop.stop()
        ## stop()


    def __onHeartBeat(self) :
        try :
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
            s.connect( (self._meta_host, self._meta_port) )
        except :
            self._logger.excpt()
            self._heart_beat_interval = 1
            deadline = time.time() + self._heart_beat_interval
            self._heart_beat_handler = self._io_loop.add_timeout(deadline, self.__onHeartBeat)
            return

        request = self.__getHeartBeatData()

        stream = iostream.IOStream(s, io_loop=self._io_loop)
        MetaConnection(stream, (self._meta_host, self._meta_port), request, self.__onHeartBeatResponse)
        ## __onHeartBeat()

    def __onHeartBeatResponse(self, data) :
        update_required = self.__handleHeartBeatResponse( data )
        if update_required :
            self._heart_beat_interval = 1
        else :
            self._heart_beat_interval = self.HEART_BEAT_INTERVAL
        deadline = time.time() + self._heart_beat_interval
        self._heart_beat_handler = self._io_loop.add_timeout(deadline, self.__onHeartBeat)
        ## __onHeartBeatResponse()

    ## class Tcpio


class   MetaConnection(object) :
    def __init__(self, stream, address, request, response_callback, end_callback=None) :
        self._stream = stream
        self._address = address
        self._response_callback = response_callback
        self._end_callback = end_callback

        self._keep_alive = False

        self._END_TAG = "\r\n\r\n"
        self._request_finished = False

        self._logger = logging.getLogger('metaconnect')
        self._logger.debug('create MetaConnection for %s', self._address)

        self.__wait_response_for = request.request_id
        request_str = codec.encode(request)
        self._stream.write( ''.join((request_str, self._END_TAG)) )
        self._stream.read_until(self._END_TAG, self.onResponse)
        ## __init__()

    def __del__(self) :
        #if self._stream.closed() :
        #    self._logger.debug('stream is closed when destroying')
        #else :
        #    self._logger.debug('stream is opened when destroying')
        self._logger.debug('destroy MetaConnection for %s', self._address)
        pass
        ## __del__()

    def onResponse(self, response_str) :
        response = codec.decode( response_str )
        if response.response_to == self.__wait_response_for :
            self._response_callback( response )

            self._stream.close()
            if self._end_callback : self._end_callback()
        else :
            # TODO : ignore other response and request
            self._stream.read_until(self._END_TAG, self.onResponse)
        ## onResponse()


    ## class MetaConnection


class   TcpServer(object) :
    def __init__(self, request_callback, io_loop) :
        self._request_callback = request_callback
        self._io_loop = io_loop

        self._socket = None
        self._logger = logging.getLogger('celtuce')
        ## __init__()

    def listen(self, port, address='') :
        assert not self._socket
        self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        flags = fcntl.fcntl(self._socket.fileno(), fcntl.F_GETFD)
        flags |= fcntl.FD_CLOEXEC
        fcntl.fcntl(self._socket.fileno(), fcntl.F_SETFD, flags)
        self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self._socket.setblocking(0)
        self._socket.bind((address, port))
        self._socket.listen(128)
        self._io_loop.add_handler(self._socket.fileno(), self._handleEvents, self._io_loop.READ)
        ## listen()

    def _handleEvents(self, fd, events):
        while True:
            try:
                connection, address = self._socket.accept()
            except socket.error, e:
                if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
                    return
                raise
            try:
                stream = iostream.IOStream(connection, io_loop=self._io_loop)
                TcpConnection(stream, address, self._request_callback)
            except:
                self._logger.excpt()
        ## _handleEvents()

    ## class TcpServer

class   TcpConnection(object) :
    def __init__(self, stream, address, request_callback) :
        self._stream = stream
        self._address = address
        self._request_callback = request_callback

        self._END_TAG = "\r\n\r\n"
        self._request_finished = False
        self._stream.read_until(self._END_TAG, self.__onRequest)

        self._keep_alive = True

        self._logger = logging.getLogger('celtuce')
        ## __init__()

    def __onRequest(self, data) :
        try :
            request = codec.decode( data )
        except :
            self._logger.error('Decode ERROR -- %s', data)
            self._logger.excpt()
            self.finish()
            return 

        response = self._request_callback(request)
        self.write(response)
        ## __onRequest()

    def write(self, data) :
        if data :
            response = codec.encode( data )
            self._stream.write(''.join((response, self._END_TAG)), self.__onWriteComplete)
        else :
            if not self._keep_alive :
                self.finish()
        ## write()

    def finish(self):
        if self._stream and not self._stream.writing():
            self.__finishRequest()

    def __onWriteComplete(self):
        if self._keep_alive :
            self._request_finished = False
            self._stream.read_until(self._END_TAG, self.__onRequest)
        else :
            self.__finishRequest()
        ## __onWriteComplete()

    def __finishRequest(self) :
        self._stream.close()
        self._stream = None
        ## __finishRequest()

    ## class TcpConnection


#######################################################################


class   TabletServer(object) :
    _HEARTBEAT_COUNT_INTERVAL = 50
    _TIMEOUT_WAIT_META_SERVER = 10

    # state of tablet
    _TABLET_STATE_READY     = 0
    _TABLET_STATE_READONLY  = 1
    _TABLET_STATE_WRITEONLY = 2

    def __init__(self, meta_url, tcp_port, path_list, footmark) :
        self._logger = logging.getLogger('celtuce')
        self._logger.info('TabletServer listening :%d ...', tcp_port)

        self._tcpio = Tcpio(self.handleRequest, tcp_port, meta_url, self.handleHeartBeatResponse, self.getHeartBeatRequest)

        self._tcp_port = tcp_port
        self._ipv4 = self.getipv4()

        self._footmark_path = footmark

        # tablet route table
        # dictionary of table
        #    table_name --> [ (r1_start_key, r1_stop_key, state),
        #                     (r2_start_key, r2_stop_key, state),
        #                      ... 
        #                   ]
        # state ::= ready | readonly | writeonly
        # list of 3-tuple(key_begin, key_end, state)
        #     
        self._route_tablet = {}
        self._route_tablet_text = ''
        self._route_tablet_digest = None

        self._heartbeat_countdown = 0

        self._id_base = id(self)
        self._id_serial = 1 
        self._id_maximum = 1024 * 1024 * 1024

        self._meta_queue = queue.Queue()

        self._storer = Storer()
        self._storer.startup( path_list )

        self.__init()
        self.__updateRouteTableText()

        ## __init__()

    def __del__() :
        self._logger.info('TabletServer is shutdowning.')
        ## __del__()

    def stop(self) :
        self._logger.info('TabletServer is shutdowning ...')
        self._logger.info('stop tcpio ...')
        self._tcpio.stop()

        self._logger.info('stop storer ...')
        if self._storer :
            self._storer.shutdown()
            self._storer = None

        self._logger.info('TabletServer stopped.')
        ## stop()

    def run(self) :
        try :
            self._tcpio.run()
        except :
            self._tcpio.stop()
        ## run()


    ####-----------------------------------------------------------####

    def __init(self) :
        self.__loadFootmark()
        ## __init()


    def __saveFootmark(self) :
        self._logger.info('save footmark to %s ...', self._footmark_path)

        fp = open(self._footmark_path, 'w', 1)

        # save [tablet]
        fp.write("[tablet]\n")
        for table_name,tablets in self._route_tablet.iteritems() :
            for k_start,k_stop,k_state in tablets :
                tablet_name = MD5(k_stop).hexdigest()[:16]
                fp.write("%s.%s = %s\n" % (table_name, tablet_name, self.__getTabletStateString(k_state)))
                fp.write("%s.%s.key_begin = %s\n" % (table_name, tablet_name, repr(k_start)))
                fp.write("%s.%s.key_end = %s\n" % (table_name, tablet_name, repr(k_stop)))
                fp.write("\n")

        # save [tablet.mark]
        fp.write("[tablet.mark]\n")
        fp.write("mark.digest = %s\n" % self._route_tablet_digest)
        fp.write("mark.data = %s\n" % self._route_tablet_text)
        fp.write("\n")

        fp.close()
        ## __saveFootmark()


    def __loadFootmark(self) :
        if not os.path.exists( self._footmark_path ) :
            return

        self._logger.debug('load footmark ...')
        ini = configparser.ConfigParser()
        ini.optionxform = lambda x: x
        ini.read(self._footmark_path)

        SECTION = 'tablet.mark'
        if not ini.has_section( SECTION ) :
            raise RuntimeError, 'Section [%s] does NOT exist in %s' % (SECTION, self._footmark_path)

        OPTION = 'mark.digest'
        self._route_tablet_digest = ini.get(SECTION, OPTION)

        OPTION = 'mark.data'
        self._route_tablet_text = ini.get(SECTION, OPTION)
        self._logger.debug('mark data : %s', self._route_tablet_text)

        self._route_tablet = codec.decodeRouteTablet( self._route_tablet_text )
        ## __loadFootmark()


    ####-----------------------------------------------------------####

    def getipv4(self) :
        ipv4 = socket.gethostbyname( socket.gethostname() )

        return ipv4
        ## getipv4()

    def __getTabletStateString(self, state) :
        if self._TABLET_STATE_READY == state :
            return 'ready'
        elif self._TABLET_STATE_READONLY == state :
            return 'readonly'
        elif self._TABLET_STATE_WRITEONLY == state :
            return 'writeonly'
        else :
            return 'unknown'
        ## __getTabletStateString()

    ####-----------------------------------------------------------####

    def __updateRouteTableText(self) :
        self._route_tablet_text = codec.encodeRouteTablet( self._route_tablet  )
        self._route_tablet_digest = MD5( self._route_tablet_text ).hexdigest()

        self.__saveFootmark()
        ## __updateRouteTableText()

    def getRouteTableText(self) :
        return self._route_tablet_text
        ## getRouteTableText()

    def getRouteTableDigest(self) :
        return self._route_tablet_digest
        ## getRouteTableDigest()

    def getHeartBeatRequest(self) :
        available_space = self._storer.getAvailableSpace()

        request = Event()
        request.request_id = self.genRequestID()
        request.op = TS_HEART_BEAT
        request.body = (self._ipv4, self._tcp_port, available_space, self._route_tablet_digest, self._route_tablet_text)
        self._logger.debug('heart-beat : %s', request.body)

        return request
        ## getHeartBeatRequest()

    ####-----------------------------------------------------------####


    ####-----------------------------------------------------------####

    def handleHeartBeatResponse(self, response) :
        self._logger.debug('HB-response : %s', response.body)

        # handle response
        heart_beat_required = False
        tasks = None
        if 'task' in response.body :
            tasks = response.body[ 'task' ]
        if tasks is None : 
            tasks = []
        elif isinstance(tasks, dict) :
            tasks = [ tasks[i] for i in range(len(tasks)) ]
        for task in tasks :
            #self._logger.debug('HB-task.body : %s', task.body)
            if isinstance(task.body, dict) :
                task.body = [ task.body[i] for i in range(len(task.body)) ]
            self._logger.debug('HB-task.body : %s', task.body)
            if TASK_CREATE_TABLET == task.op :
                _table,_key_start,_key_stop = task.body
                if not self.existTabletInRouteTable(_table, _key_start, _key_stop) :
                    self._logger.info('create tablet %s(%s .. %s] ...', str(_table), repr(_key_start), repr(_key_stop))
                    task_response = self._storer.createTablet(_table, _key_start, _key_stop)
                    if not task_response.isOk() :
                        self._logger.error('fail to create tablet %s(%s .. %s] -- [%d] %s', str(_table), repr(_key_start), repr(_key_stop), task_response.status_code, task_response.error)
                    else :
                        self.__addTabletToRouteTable(_table, _key_start, _key_stop)
                        heart_beat_required = True

            elif TASK_DROP_TABLET == task.op :
                _table,_key_start,_key_stop = task.body
                if self.existTabletInRouteTable(_table, _key_start, _key_stop) :
                    self._logger.info('drop tablet %s(%s .. %s] ...', str(_table), repr(_key_start), repr(_key_stop))
                    task_response = self._storer.dropTablet(_table, _key_start, _key_stop)
                    if not task_response.isOk() :
                        self._logger.error('fail to drop tablet %s(%s .. %s] -- [%d] %s', str(_table), repr(_key_start), repr(_key_stop), task_response.status_code, task_response.error)
                    else :
                        self.__delTabletFromRouteTable(_table, _key_start, _key_stop)
                        heart_beat_required = True

            else :
                self._logger.error('WHY not handle HB-task %s', str(task))

        return heart_beat_required
        ## handleHeartBeatResponse()

    def __addTabletToRouteTable(self, table_name, key_start, key_stop) :
        if table_name not in self._route_tablet :
            self._route_tablet[ table_name ] = [(key_start, key_stop, self._TABLET_STATE_READY),]
            self.__updateRouteTableText()
            return True

        self._logger.error('TODO : to add tablet to route table')
        return True
        ## __addTabletToRouteTable()

    def __delTabletFromRouteTable(self, table_name, key_start, key_stop) :
        self._logger.error('TODO : to delete tablet from route table')
        ## __delTabletFromRouteTable()

    def existTabletInRouteTable(self, table_name, key_start, key_stop) :
        if table_name not in self._route_tablet :
            return False

        tablets = self._route_tablet[ table_name ]
        index = bisect_left(tablets, (key_start,key_stop,0))
        if index >= len(tablets) :
            return False
        if index > 0 :
            index -= 1

        tablet = tablets[ index ]
        if key_start == tablet[0] and key_stop == tablet[1] :
            return True

        return False
        ## existTabletInRouteTable()

    ####-----------------------------------------------------------####


    def genRequestID(self) :
        # TODO: lock
        serial = self._id_serial

        self._id_serial += 1
        if self._id_serial >= self._id_maximum :
            self._id_serial = 1 

        id = '.'.join((str(self._id_base), str(serial)))
        return id
        ## genRequestID()


    ####-----------------------------------------------------------####

    def handleRequest(self, request) :

        request = request
        self._logger.debug('TabletServer::handleRequest %s', request)
        #self._logger.debug('TabletServer::handleRequest body %s', request.body)

        response = Event()
        response.request_id = self.genRequestID()
        response.op = OP_RESPONSE
        response.response_to = request.request_id
        if OP_INSERT == request.op :
            if isinstance(request.body, dict) :
                request.body = [ request.body[i] for i in range(len(request.body)) ]
            _table,_key,_data,_timestamp_ms = request.body
            _response = self._storer.insert(_table, _key, _data, _timestamp_ms)
            response.body = _response
        elif OP_INSERTUNIQUE == request.op :
            if isinstance(request.body, dict) :
                request.body = [ request.body[i] for i in range(len(request.body)) ]
            _table,_key,_data,_timestamp_ms = request.body
            _response = self._storer.insertunique(_table, _key, _data, _timestamp_ms)
            response.body = _response
        elif OP_FIND == request.op :
            if isinstance(request.body, dict) :
                request.body = [ request.body[i] for i in range(len(request.body)) ]
            _table,_key,_timestamp_ms = request.body
            _response = self._storer.find(_table, _key, _timestamp_ms)
            response.body = _response
        elif OP_ERASE == request.op :
            if isinstance(request.body, dict) :
                request.body = [ request.body[i] for i in range(len(request.body)) ]
            _table,_key,_timestamp_ms = request.body
            _response = self._storer.erase(_table, _key, _timestamp_ms)
            response.body = _response

        elif OP_TEST_ECHO == request.op :
            response.body = request.body

        else :
            raise RuntimeError,'Why not handle event "%s"' % str(request)

        return response
        ## handleRequest()

    ## class TabletServer


#######################################################################

def parseTabletServerConfig( config_file ) :
    import ConfigParser as configparser

    ini = configparser.ConfigParser()
    ini.optionxform = lambda x: x
    ini.read( config_file )

    SECTION = 'tablet.server'
    if not ini.has_section(SECTION) :
        raise RuntimeError, 'Section [%s] does NOT exist in %s' % (SECTION, config_file)

    OPTION = 'port'
    if not ini.has_option(SECTION, OPTION) :
        raise RuntimeError, 'Section [%s] has no option "%s" in %s' % (SECTION, OPTION, config_file)
    port = ini.getint(SECTION, OPTION)

    OPTION = 'metaserver'
    if not ini.has_option(SECTION, OPTION) :
        raise RuntimeError, 'Section [%s] has no option "%s" in %s' % (SECTION, OPTION, config_file)
    meta_url = ini.get(SECTION, OPTION)

    OPTION = 'path_list'
    if not ini.has_option(SECTION, OPTION) :
        raise RuntimeError, 'Section [%s] has no option "%s" in %s' % (SECTION, OPTION, config_file)
    path_list = ini.get(SECTION, OPTION)

    OPTION = 'footmark'
    if not ini.has_option(SECTION, OPTION) :
        raise RuntimeError, 'Section [%s] has no option "%s" in %s' % (SECTION, OPTION, config_file)
    footmark = ini.get(SECTION, OPTION)

    cfg = type('TabletServerConfig',(object,),{})
    cfg.port = port
    cfg.meta_url = meta_url
    cfg.path_list = path_list
    cfg.footmark = footmark

    return cfg
    ## parseTabletServerConfig()


if __name__ == '__main__' :

    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option("-c", "--config", dest="config", type="string", 
                        help='path to configure file of tablet-server')
    parser.add_option("-l", "--logconf", dest="logconfig", type="string",
                        help='path to logging configure file')

    parser.add_option("-d", "--debug", dest="debug", action="store_true", default=False,
                        help='show debug information')
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
                        help='show detail information')

    (options, args) = parser.parse_args()
    if not options.config :
        parser.print_help( sys.stdout )
        sys.exit(0)

    if options.logconfig :
        logging.loadConfigure( options.logconfig )

    if options.debug :
        logger = logging.getLogger()
        logger.setLevel(logging.LEVEL_DEBUG)

    cfg = parseTabletServerConfig( options.config )

    # check path list
    import os.path,sys
    paths_ok = True
    paths = cfg.path_list.split(':')
    for path in paths :
        if not path : continue
        if not os.path.exists(path) :
            paths_ok = False
            sys.stderr.write('ERROR : path %s does NOT exist.\n' % repr(path))
        if not os.path.isdir(path) :
            paths_ok = False
            sys.stderr.write('ERROR : path %s is NOT a directory.\n' % repr(path))
    if not paths_ok :
        sys.exit(1)

    server = TabletServer(cfg.meta_url, cfg.port, cfg.path_list, cfg.footmark)
    
    try :
        server.run()
    finally :
        server.stop()

    sys.exit(0)

